from hdfs import Client
import datetime
import configparser
import os
from string import Template

conf = configparser.ConfigParser()
conf.read('./config.ini')
cli = Client(url=conf.get('all', 'url'))
os.makedirs('./all', exist_ok=True)
os.makedirs('./warn', exist_ok=True)
os.makedirs('./templates', exist_ok=True)
tables = [_ for _ in conf.keys() if 'table' in _]
tr = Template('<tr><td>$table</td><td>$time</td><td>$except</td><td>$practical</td><td>$partition</td></tr>')


def handle_size(table):
    path = handle_path(conf.get(table, 'path'))
    size = None
    if cli.status(path, strict=False) is not None:
        size = get_file_size(path=path)
    return size, path


# all file
# 时间格式 YYYYmmddHHMMSS
# 目录的实际大小(byte),目录的实际大小(MB),目录的实际大小(GB),执行开始的时间,执行结束的时间,是否告警(1:告警,0:不告警),监控的文件目录
def main():
    template_line = []
    for table in tables:
        start_time = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
        with open('./all/{}.log'.format(table), mode='a', encoding='utf-8') as file:
            size_b, path = handle_size(table)
            end_time = datetime.datetime.now()
            hour = end_time.hour
            end_time = end_time.strftime('%Y%m%d%H%M%S')
            size_g = human(size_b, 'G')
            warning = warn(table, size_b)
            partitions = '-'.join(path.split('/')[-3:])
            line = '{},{},{},{},{},{}\n'.format(table, size_g, start_time, end_time, warning, partitions)
            file.write(line)

            if warning:
                # 写入一个head 写入一条tr
                keys = {
                    'table': table,
                    'time': end_time,
                    'except': conf.get(table, 'target_size').split(',')[hour],
                    'practical': size_g,
                    'partition': partitions
                }
                template_line.append(keys)
                warn_file_name = './warn/{}.log'.format(table)
                with open(warn_file_name, mode='a', encoding='utf-8') as warn_file:
                    warn_file.write(line)
    handle_template(template_line)


def handle_template(line):
    if len(line) == 0:
        return
    now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
    file_name = '{}.html'.format(now)
    with open(file_name, mode='w', encoding='utf-8') as file:
        file.write('表名\t运行时间\t预期大小\t实际大小\t分区标识\n')
        for l in line:
            file.write('{}\t'.format(l['table']))
            file.write('{}\t'.format(l['time']))
            file.write('{}\t'.format(l['except']))
            file.write('{}\t'.format(l['practical']))
            file.write('{}\t'.format(l['partition']))
            file.write('\n')
    send2dmz(file_name)


def send2dmz(file):
    # 将产生的文件发送给dmz
    os.system('expect scp.exp ./{} /sc_bin/check_hdfs_size/hdfs_size/'.format(file))
    os.system('mv ./{} ./templates/'.format(file))


def warn(table, size):
    hour = datetime.datetime.now().hour
    target_size = unhuman(conf.get(table, 'target_size').split(',')[hour])
    floor = unhuman(conf.get(table, 'floor'))
    upper = unhuman(conf.get(table, 'upper'))
    return 0 if target_size - floor <= size <= target_size + upper else 1


def handle_path(path):
    '''
    获取当前的时间 - 一个小时 作为处理目录的传入值
    :param path:
    :return:
    '''
    now = (datetime.datetime.now() - datetime.timedelta(hours=1)).strftime('%Y%m%d%H')
    return path.format(now[:6], now[:8], now)


def unhuman(num_str):
    unit = num_str[-1]
    num = float(num_str[:-1])
    if unit == 'G':
        num = num * 1024 ** 3
    elif unit == 'M':
        num = num * 1024 ** 2
    return round(num)


def human(num, unit):
    '''
    将传入的字节数处理成人类可见的方式
    :param num: 需要处理的byte数字 (int)
    :param unit: 需要转成的单位 M G (str)
    :return: str 转换成的字符
    '''
    if unit == 'M':
        num = num / (1024 ** 2)
    elif unit == 'G':
        num = num / (1024 ** 3)
    else:
        return None

    return '{}{}'.format(round(num, 2), unit)


def get_file_size(path):
    '''
    递归统计一个目录的大小
    :param path: 需要递归处理的处理
    :return: 字节大小 byte
    '''
    status = cli.status(path)
    if status is not None:
        size = int(status['length'])
        if status['childrenNum'] > 0 and status['type'] == 'DIRECTORY':
            childs = [get_file_size('{}/{}'.format(path, _)) for _ in cli.list(path)]
            size = size + sum(childs)
        return size
